{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Installation and Imports\n",
    "Please follow the **installation guide** in the [ThoughtSource Readme file](https://github.com/OpenBioLink/ThoughtSource) before using this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from cot import Collection\n",
    "from cot.generate import FRAGMENTS\n",
    "from rich.pretty import pprint\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100 = Collection.load_thoughtsource_100()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100.select_generated_cots(model = \"gpt-3.5-turbo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100.select_generated_cots(cot_trigger = None, instruction = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['commonsense_qa.validation.accuracy.gpt-3.5-turbo.None_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.None_None_kojima-A-E', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.None_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.None_None_kojima-A-D', 'strategy_qa.train.accuracy.gpt-3.5-turbo.None_None_kojima-yes-no', 'worldtree.test.accuracy.gpt-3.5-turbo.None_None_kojima-A-D']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_e177d_row0_col0, #T_e177d_row1_col0, #T_e177d_row2_col0, #T_e177d_row3_col0, #T_e177d_row4_col0, #T_e177d_row5_col0, #T_e177d_row6_col0 {\n",
       "  font-weight: bold;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_e177d\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_e177d_level0_col0\" class=\"col_heading level0 col0\" >None_None</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"blank level1\" >&nbsp;</th>\n",
       "      <th id=\"T_e177d_level1_col0\" class=\"col_heading level1 col0\" >gpt-3.5-turbo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row0\" class=\"row_heading level0 row0\" >commonsense_qa</th>\n",
       "      <td id=\"T_e177d_row0_col0\" class=\"data row0 col0\" >0.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row1\" class=\"row_heading level0 row1\" >med_qa</th>\n",
       "      <td id=\"T_e177d_row1_col0\" class=\"data row1 col0\" >0.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row2\" class=\"row_heading level0 row2\" >medmc_qa</th>\n",
       "      <td id=\"T_e177d_row2_col0\" class=\"data row2 col0\" >0.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row3\" class=\"row_heading level0 row3\" >open_book_qa</th>\n",
       "      <td id=\"T_e177d_row3_col0\" class=\"data row3 col0\" >0.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row4\" class=\"row_heading level0 row4\" >strategy_qa</th>\n",
       "      <td id=\"T_e177d_row4_col0\" class=\"data row4 col0\" >0.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row5\" class=\"row_heading level0 row5\" >worldtree</th>\n",
       "      <td id=\"T_e177d_row5_col0\" class=\"data row5 col0\" >0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_e177d_level0_row6\" class=\"row_heading level0 row6\" >Average</th>\n",
       "      <td id=\"T_e177d_row6_col0\" class=\"data row6 col0\" >0.70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8c6d4d7a30>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval = ts_100.evaluate()\n",
    "evaluation_as_table(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100[\"worldtree\"][\"test\"][\"generated_cot\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_100.select_generated_cots(model = \"gpt-4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot.stats import evaluation_as_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['commonsense_qa.validation.accuracy.gpt-3.5-turbo.None_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.None_kojima-01_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.None_kojima-03_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.None_kojima-09_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.None_zhou-01_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-01_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-05_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-08_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-09_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-10_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-12_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-13_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-16_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.qa-17_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.zhou-01-ins_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-3.5-turbo.zhou-01-ins_zhou-01_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.None_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.None_kojima-01_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.None_kojima-03_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.None_kojima-09_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.None_zhou-01_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-01_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-05_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-08_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-09_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-10_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-12_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-13_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-16_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.qa-17_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.zhou-01-ins_None_kojima-A-E', 'med_qa.test.accuracy.gpt-3.5-turbo.zhou-01-ins_zhou-01_kojima-A-E', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.None_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.None_kojima-01_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.None_kojima-03_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.None_kojima-09_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.None_zhou-01_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-01_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-05_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-08_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-09_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-10_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-12_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-13_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-16_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.qa-17_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.zhou-01-ins_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-3.5-turbo.zhou-01-ins_zhou-01_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.None_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.None_kojima-01_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.None_kojima-03_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.None_kojima-09_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.None_zhou-01_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-01_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-05_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-08_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-09_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-10_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-12_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-13_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-16_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.qa-17_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.zhou-01-ins_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-3.5-turbo.zhou-01-ins_zhou-01_kojima-A-D', 'strategy_qa.train.accuracy.gpt-3.5-turbo.None_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.None_kojima-01_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.None_kojima-03_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.None_kojima-09_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.None_zhou-01_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-01_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-05_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-08_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-09_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-10_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-12_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-13_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-16_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.qa-17_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.zhou-01-ins_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-3.5-turbo.zhou-01-ins_zhou-01_kojima-yes-no', 'worldtree.test.accuracy.gpt-3.5-turbo.None_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.None_kojima-01_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.None_kojima-03_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.None_kojima-09_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.None_zhou-01_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-01_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-05_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-08_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-09_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-10_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-12_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-13_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-16_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.qa-17_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.zhou-01-ins_None_kojima-A-D', 'worldtree.test.accuracy.gpt-3.5-turbo.zhou-01-ins_zhou-01_kojima-A-D']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_709bb_row0_col0, #T_709bb_row1_col4, #T_709bb_row2_col0, #T_709bb_row3_col4, #T_709bb_row4_col11, #T_709bb_row5_col0, #T_709bb_row6_col0 {\n",
       "  font-weight: bold;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_709bb\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_709bb_level0_col0\" class=\"col_heading level0 col0\" >None_None</th>\n",
       "      <th id=\"T_709bb_level0_col1\" class=\"col_heading level0 col1\" >None_kojima-01</th>\n",
       "      <th id=\"T_709bb_level0_col2\" class=\"col_heading level0 col2\" >None_kojima-03</th>\n",
       "      <th id=\"T_709bb_level0_col3\" class=\"col_heading level0 col3\" >None_kojima-09</th>\n",
       "      <th id=\"T_709bb_level0_col4\" class=\"col_heading level0 col4\" >None_zhou-01</th>\n",
       "      <th id=\"T_709bb_level0_col5\" class=\"col_heading level0 col5\" >qa-01_None</th>\n",
       "      <th id=\"T_709bb_level0_col6\" class=\"col_heading level0 col6\" >qa-05_None</th>\n",
       "      <th id=\"T_709bb_level0_col7\" class=\"col_heading level0 col7\" >qa-08_None</th>\n",
       "      <th id=\"T_709bb_level0_col8\" class=\"col_heading level0 col8\" >qa-09_None</th>\n",
       "      <th id=\"T_709bb_level0_col9\" class=\"col_heading level0 col9\" >qa-10_None</th>\n",
       "      <th id=\"T_709bb_level0_col10\" class=\"col_heading level0 col10\" >qa-12_None</th>\n",
       "      <th id=\"T_709bb_level0_col11\" class=\"col_heading level0 col11\" >qa-13_None</th>\n",
       "      <th id=\"T_709bb_level0_col12\" class=\"col_heading level0 col12\" >qa-16_None</th>\n",
       "      <th id=\"T_709bb_level0_col13\" class=\"col_heading level0 col13\" >qa-17_None</th>\n",
       "      <th id=\"T_709bb_level0_col14\" class=\"col_heading level0 col14\" >zhou-01-ins_None</th>\n",
       "      <th id=\"T_709bb_level0_col15\" class=\"col_heading level0 col15\" >zhou-01-ins_zhou-01</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"blank level1\" >&nbsp;</th>\n",
       "      <th id=\"T_709bb_level1_col0\" class=\"col_heading level1 col0\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col1\" class=\"col_heading level1 col1\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col2\" class=\"col_heading level1 col2\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col3\" class=\"col_heading level1 col3\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col4\" class=\"col_heading level1 col4\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col5\" class=\"col_heading level1 col5\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col6\" class=\"col_heading level1 col6\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col7\" class=\"col_heading level1 col7\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col8\" class=\"col_heading level1 col8\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col9\" class=\"col_heading level1 col9\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col10\" class=\"col_heading level1 col10\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col11\" class=\"col_heading level1 col11\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col12\" class=\"col_heading level1 col12\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col13\" class=\"col_heading level1 col13\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col14\" class=\"col_heading level1 col14\" >gpt-3.5-turbo</th>\n",
       "      <th id=\"T_709bb_level1_col15\" class=\"col_heading level1 col15\" >gpt-3.5-turbo</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row0\" class=\"row_heading level0 row0\" >commonsense_qa</th>\n",
       "      <td id=\"T_709bb_row0_col0\" class=\"data row0 col0\" >0.72</td>\n",
       "      <td id=\"T_709bb_row0_col1\" class=\"data row0 col1\" >0.67</td>\n",
       "      <td id=\"T_709bb_row0_col2\" class=\"data row0 col2\" >0.63</td>\n",
       "      <td id=\"T_709bb_row0_col3\" class=\"data row0 col3\" >0.70</td>\n",
       "      <td id=\"T_709bb_row0_col4\" class=\"data row0 col4\" >0.66</td>\n",
       "      <td id=\"T_709bb_row0_col5\" class=\"data row0 col5\" >0.66</td>\n",
       "      <td id=\"T_709bb_row0_col6\" class=\"data row0 col6\" >0.69</td>\n",
       "      <td id=\"T_709bb_row0_col7\" class=\"data row0 col7\" >0.62</td>\n",
       "      <td id=\"T_709bb_row0_col8\" class=\"data row0 col8\" >0.64</td>\n",
       "      <td id=\"T_709bb_row0_col9\" class=\"data row0 col9\" >0.68</td>\n",
       "      <td id=\"T_709bb_row0_col10\" class=\"data row0 col10\" >0.63</td>\n",
       "      <td id=\"T_709bb_row0_col11\" class=\"data row0 col11\" >0.61</td>\n",
       "      <td id=\"T_709bb_row0_col12\" class=\"data row0 col12\" >0.58</td>\n",
       "      <td id=\"T_709bb_row0_col13\" class=\"data row0 col13\" >0.66</td>\n",
       "      <td id=\"T_709bb_row0_col14\" class=\"data row0 col14\" >0.72</td>\n",
       "      <td id=\"T_709bb_row0_col15\" class=\"data row0 col15\" >0.65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row1\" class=\"row_heading level0 row1\" >med_qa</th>\n",
       "      <td id=\"T_709bb_row1_col0\" class=\"data row1 col0\" >0.58</td>\n",
       "      <td id=\"T_709bb_row1_col1\" class=\"data row1 col1\" >0.59</td>\n",
       "      <td id=\"T_709bb_row1_col2\" class=\"data row1 col2\" >0.59</td>\n",
       "      <td id=\"T_709bb_row1_col3\" class=\"data row1 col3\" >0.51</td>\n",
       "      <td id=\"T_709bb_row1_col4\" class=\"data row1 col4\" >0.65</td>\n",
       "      <td id=\"T_709bb_row1_col5\" class=\"data row1 col5\" >0.54</td>\n",
       "      <td id=\"T_709bb_row1_col6\" class=\"data row1 col6\" >0.53</td>\n",
       "      <td id=\"T_709bb_row1_col7\" class=\"data row1 col7\" >0.46</td>\n",
       "      <td id=\"T_709bb_row1_col8\" class=\"data row1 col8\" >0.55</td>\n",
       "      <td id=\"T_709bb_row1_col9\" class=\"data row1 col9\" >0.56</td>\n",
       "      <td id=\"T_709bb_row1_col10\" class=\"data row1 col10\" >0.59</td>\n",
       "      <td id=\"T_709bb_row1_col11\" class=\"data row1 col11\" >0.49</td>\n",
       "      <td id=\"T_709bb_row1_col12\" class=\"data row1 col12\" >0.60</td>\n",
       "      <td id=\"T_709bb_row1_col13\" class=\"data row1 col13\" >0.56</td>\n",
       "      <td id=\"T_709bb_row1_col14\" class=\"data row1 col14\" >0.54</td>\n",
       "      <td id=\"T_709bb_row1_col15\" class=\"data row1 col15\" >0.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row2\" class=\"row_heading level0 row2\" >medmc_qa</th>\n",
       "      <td id=\"T_709bb_row2_col0\" class=\"data row2 col0\" >0.58</td>\n",
       "      <td id=\"T_709bb_row2_col1\" class=\"data row2 col1\" >0.47</td>\n",
       "      <td id=\"T_709bb_row2_col2\" class=\"data row2 col2\" >0.50</td>\n",
       "      <td id=\"T_709bb_row2_col3\" class=\"data row2 col3\" >0.50</td>\n",
       "      <td id=\"T_709bb_row2_col4\" class=\"data row2 col4\" >0.48</td>\n",
       "      <td id=\"T_709bb_row2_col5\" class=\"data row2 col5\" >0.47</td>\n",
       "      <td id=\"T_709bb_row2_col6\" class=\"data row2 col6\" >0.42</td>\n",
       "      <td id=\"T_709bb_row2_col7\" class=\"data row2 col7\" >0.45</td>\n",
       "      <td id=\"T_709bb_row2_col8\" class=\"data row2 col8\" >0.47</td>\n",
       "      <td id=\"T_709bb_row2_col9\" class=\"data row2 col9\" >0.49</td>\n",
       "      <td id=\"T_709bb_row2_col10\" class=\"data row2 col10\" >0.53</td>\n",
       "      <td id=\"T_709bb_row2_col11\" class=\"data row2 col11\" >0.41</td>\n",
       "      <td id=\"T_709bb_row2_col12\" class=\"data row2 col12\" >0.48</td>\n",
       "      <td id=\"T_709bb_row2_col13\" class=\"data row2 col13\" >0.53</td>\n",
       "      <td id=\"T_709bb_row2_col14\" class=\"data row2 col14\" >0.44</td>\n",
       "      <td id=\"T_709bb_row2_col15\" class=\"data row2 col15\" >0.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row3\" class=\"row_heading level0 row3\" >open_book_qa</th>\n",
       "      <td id=\"T_709bb_row3_col0\" class=\"data row3 col0\" >0.77</td>\n",
       "      <td id=\"T_709bb_row3_col1\" class=\"data row3 col1\" >0.77</td>\n",
       "      <td id=\"T_709bb_row3_col2\" class=\"data row3 col2\" >0.73</td>\n",
       "      <td id=\"T_709bb_row3_col3\" class=\"data row3 col3\" >0.73</td>\n",
       "      <td id=\"T_709bb_row3_col4\" class=\"data row3 col4\" >0.81</td>\n",
       "      <td id=\"T_709bb_row3_col5\" class=\"data row3 col5\" >0.73</td>\n",
       "      <td id=\"T_709bb_row3_col6\" class=\"data row3 col6\" >0.65</td>\n",
       "      <td id=\"T_709bb_row3_col7\" class=\"data row3 col7\" >0.73</td>\n",
       "      <td id=\"T_709bb_row3_col8\" class=\"data row3 col8\" >0.71</td>\n",
       "      <td id=\"T_709bb_row3_col9\" class=\"data row3 col9\" >0.73</td>\n",
       "      <td id=\"T_709bb_row3_col10\" class=\"data row3 col10\" >0.80</td>\n",
       "      <td id=\"T_709bb_row3_col11\" class=\"data row3 col11\" >0.72</td>\n",
       "      <td id=\"T_709bb_row3_col12\" class=\"data row3 col12\" >0.69</td>\n",
       "      <td id=\"T_709bb_row3_col13\" class=\"data row3 col13\" >0.69</td>\n",
       "      <td id=\"T_709bb_row3_col14\" class=\"data row3 col14\" >0.76</td>\n",
       "      <td id=\"T_709bb_row3_col15\" class=\"data row3 col15\" >0.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row4\" class=\"row_heading level0 row4\" >strategy_qa</th>\n",
       "      <td id=\"T_709bb_row4_col0\" class=\"data row4 col0\" >0.57</td>\n",
       "      <td id=\"T_709bb_row4_col1\" class=\"data row4 col1\" >0.56</td>\n",
       "      <td id=\"T_709bb_row4_col2\" class=\"data row4 col2\" >0.52</td>\n",
       "      <td id=\"T_709bb_row4_col3\" class=\"data row4 col3\" >0.58</td>\n",
       "      <td id=\"T_709bb_row4_col4\" class=\"data row4 col4\" >0.59</td>\n",
       "      <td id=\"T_709bb_row4_col5\" class=\"data row4 col5\" >0.44</td>\n",
       "      <td id=\"T_709bb_row4_col6\" class=\"data row4 col6\" >0.43</td>\n",
       "      <td id=\"T_709bb_row4_col7\" class=\"data row4 col7\" >0.58</td>\n",
       "      <td id=\"T_709bb_row4_col8\" class=\"data row4 col8\" >0.62</td>\n",
       "      <td id=\"T_709bb_row4_col9\" class=\"data row4 col9\" >0.56</td>\n",
       "      <td id=\"T_709bb_row4_col10\" class=\"data row4 col10\" >0.50</td>\n",
       "      <td id=\"T_709bb_row4_col11\" class=\"data row4 col11\" >0.64</td>\n",
       "      <td id=\"T_709bb_row4_col12\" class=\"data row4 col12\" >0.63</td>\n",
       "      <td id=\"T_709bb_row4_col13\" class=\"data row4 col13\" >0.58</td>\n",
       "      <td id=\"T_709bb_row4_col14\" class=\"data row4 col14\" >0.52</td>\n",
       "      <td id=\"T_709bb_row4_col15\" class=\"data row4 col15\" >0.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row5\" class=\"row_heading level0 row5\" >worldtree</th>\n",
       "      <td id=\"T_709bb_row5_col0\" class=\"data row5 col0\" >0.96</td>\n",
       "      <td id=\"T_709bb_row5_col1\" class=\"data row5 col1\" >0.93</td>\n",
       "      <td id=\"T_709bb_row5_col2\" class=\"data row5 col2\" >0.95</td>\n",
       "      <td id=\"T_709bb_row5_col3\" class=\"data row5 col3\" >0.95</td>\n",
       "      <td id=\"T_709bb_row5_col4\" class=\"data row5 col4\" >0.92</td>\n",
       "      <td id=\"T_709bb_row5_col5\" class=\"data row5 col5\" >0.95</td>\n",
       "      <td id=\"T_709bb_row5_col6\" class=\"data row5 col6\" >0.74</td>\n",
       "      <td id=\"T_709bb_row5_col7\" class=\"data row5 col7\" >0.92</td>\n",
       "      <td id=\"T_709bb_row5_col8\" class=\"data row5 col8\" >0.91</td>\n",
       "      <td id=\"T_709bb_row5_col9\" class=\"data row5 col9\" >0.95</td>\n",
       "      <td id=\"T_709bb_row5_col10\" class=\"data row5 col10\" >0.92</td>\n",
       "      <td id=\"T_709bb_row5_col11\" class=\"data row5 col11\" >0.96</td>\n",
       "      <td id=\"T_709bb_row5_col12\" class=\"data row5 col12\" >0.91</td>\n",
       "      <td id=\"T_709bb_row5_col13\" class=\"data row5 col13\" >0.92</td>\n",
       "      <td id=\"T_709bb_row5_col14\" class=\"data row5 col14\" >0.96</td>\n",
       "      <td id=\"T_709bb_row5_col15\" class=\"data row5 col15\" >0.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_709bb_level0_row6\" class=\"row_heading level0 row6\" >Average</th>\n",
       "      <td id=\"T_709bb_row6_col0\" class=\"data row6 col0\" >0.70</td>\n",
       "      <td id=\"T_709bb_row6_col1\" class=\"data row6 col1\" >0.66</td>\n",
       "      <td id=\"T_709bb_row6_col2\" class=\"data row6 col2\" >0.65</td>\n",
       "      <td id=\"T_709bb_row6_col3\" class=\"data row6 col3\" >0.66</td>\n",
       "      <td id=\"T_709bb_row6_col4\" class=\"data row6 col4\" >0.68</td>\n",
       "      <td id=\"T_709bb_row6_col5\" class=\"data row6 col5\" >0.63</td>\n",
       "      <td id=\"T_709bb_row6_col6\" class=\"data row6 col6\" >0.58</td>\n",
       "      <td id=\"T_709bb_row6_col7\" class=\"data row6 col7\" >0.63</td>\n",
       "      <td id=\"T_709bb_row6_col8\" class=\"data row6 col8\" >0.65</td>\n",
       "      <td id=\"T_709bb_row6_col9\" class=\"data row6 col9\" >0.66</td>\n",
       "      <td id=\"T_709bb_row6_col10\" class=\"data row6 col10\" >0.66</td>\n",
       "      <td id=\"T_709bb_row6_col11\" class=\"data row6 col11\" >0.64</td>\n",
       "      <td id=\"T_709bb_row6_col12\" class=\"data row6 col12\" >0.65</td>\n",
       "      <td id=\"T_709bb_row6_col13\" class=\"data row6 col13\" >0.66</td>\n",
       "      <td id=\"T_709bb_row6_col14\" class=\"data row6 col14\" >0.66</td>\n",
       "      <td id=\"T_709bb_row6_col15\" class=\"data row6 col15\" >0.64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8c82475cf0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval = ts_100.evaluate()\n",
    "evaluation_as_table(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['commonsense_qa.validation.accuracy.gpt-4.None_None_kojima-A-E', 'commonsense_qa.validation.accuracy.gpt-4.None_zhou-01_kojima-A-E', 'med_qa.test.accuracy.gpt-4.None_None_kojima-A-E', 'med_qa.test.accuracy.gpt-4.None_zhou-01_kojima-A-E', 'medmc_qa.validation.accuracy.gpt-4.None_None_kojima-A-D', 'medmc_qa.validation.accuracy.gpt-4.None_zhou-01_kojima-A-D', 'open_book_qa.test.accuracy.gpt-4.None_None_kojima-A-D', 'open_book_qa.test.accuracy.gpt-4.None_zhou-01_kojima-A-D', 'strategy_qa.train.accuracy.gpt-4.None_None_kojima-yes-no', 'strategy_qa.train.accuracy.gpt-4.None_zhou-01_kojima-yes-no', 'worldtree.test.accuracy.gpt-4.None_None_kojima-A-D', 'worldtree.test.accuracy.gpt-4.None_zhou-01_kojima-A-D']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_c09b5_row0_col0, #T_c09b5_row1_col1, #T_c09b5_row2_col1, #T_c09b5_row3_col1, #T_c09b5_row4_col1, #T_c09b5_row5_col0, #T_c09b5_row6_col1 {\n",
       "  font-weight: bold;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_c09b5\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_c09b5_level0_col0\" class=\"col_heading level0 col0\" >None_None</th>\n",
       "      <th id=\"T_c09b5_level0_col1\" class=\"col_heading level0 col1\" >None_zhou-01</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"blank level1\" >&nbsp;</th>\n",
       "      <th id=\"T_c09b5_level1_col0\" class=\"col_heading level1 col0\" >gpt-4</th>\n",
       "      <th id=\"T_c09b5_level1_col1\" class=\"col_heading level1 col1\" >gpt-4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row0\" class=\"row_heading level0 row0\" >commonsense_qa</th>\n",
       "      <td id=\"T_c09b5_row0_col0\" class=\"data row0 col0\" >0.75</td>\n",
       "      <td id=\"T_c09b5_row0_col1\" class=\"data row0 col1\" >0.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row1\" class=\"row_heading level0 row1\" >med_qa</th>\n",
       "      <td id=\"T_c09b5_row1_col0\" class=\"data row1 col0\" >0.73</td>\n",
       "      <td id=\"T_c09b5_row1_col1\" class=\"data row1 col1\" >0.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row2\" class=\"row_heading level0 row2\" >medmc_qa</th>\n",
       "      <td id=\"T_c09b5_row2_col0\" class=\"data row2 col0\" >0.69</td>\n",
       "      <td id=\"T_c09b5_row2_col1\" class=\"data row2 col1\" >0.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row3\" class=\"row_heading level0 row3\" >open_book_qa</th>\n",
       "      <td id=\"T_c09b5_row3_col0\" class=\"data row3 col0\" >0.92</td>\n",
       "      <td id=\"T_c09b5_row3_col1\" class=\"data row3 col1\" >0.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row4\" class=\"row_heading level0 row4\" >strategy_qa</th>\n",
       "      <td id=\"T_c09b5_row4_col0\" class=\"data row4 col0\" >0.71</td>\n",
       "      <td id=\"T_c09b5_row4_col1\" class=\"data row4 col1\" >0.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row5\" class=\"row_heading level0 row5\" >worldtree</th>\n",
       "      <td id=\"T_c09b5_row5_col0\" class=\"data row5 col0\" >0.99</td>\n",
       "      <td id=\"T_c09b5_row5_col1\" class=\"data row5 col1\" >0.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_c09b5_level0_row6\" class=\"row_heading level0 row6\" >Average</th>\n",
       "      <td id=\"T_c09b5_row6_col0\" class=\"data row6 col0\" >0.80</td>\n",
       "      <td id=\"T_c09b5_row6_col1\" class=\"data row6 col1\" >0.82</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f8d081acf10>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval = ts_100.evaluate()\n",
    "evaluation_as_table(eval)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select where GPT is wrong with direct prompting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.load_thoughtsource_100()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll.select_generated_cots(\n",
    "    author=\n",
    "        'thoughtsource',\n",
    "    model=[\n",
    "        # 'gpt-3.5-turbo',\n",
    "        'gpt-4',\n",
    "        # 'flan-T5-xxl',\n",
    "    ],\n",
    "    cot_trigger=[\n",
    "        None, \n",
    "        # 'zhou-01'\n",
    "    ],\n",
    "    instruction=[\n",
    "        None\n",
    "    ],\n",
    "    answer= False\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 25      | -      |\n",
       "| med_qa         | -       | -       | 27     |\n",
       "| medmc_qa       | -       | 31      | -      |\n",
       "| open_book_qa   | -       | -       | 8      |\n",
       "| strategy_qa    | 29      | -       | -      |\n",
       "| worldtree      | -       | -       | 1      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll = coll.filter(lambda x: len(x[\"generated_cot\"])==1)\n",
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll_1 = coll.copy()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select where GPT is right with CoT prompting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.load_thoughtsource_100()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll.select_generated_cots(\n",
    "    author=\n",
    "        'thoughtsource',\n",
    "    model=[\n",
    "        # 'gpt-3.5-turbo',\n",
    "        'gpt-4',\n",
    "        # 'flan-T5-xxl',\n",
    "    ],\n",
    "    cot_trigger=[\n",
    "        # None, \n",
    "        'zhou-01'\n",
    "    ],\n",
    "    instruction=[\n",
    "        None\n",
    "    ],\n",
    "    answer= True\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 72      | -      |\n",
       "| med_qa         | -       | -       | 76     |\n",
       "| medmc_qa       | -       | 70      | -      |\n",
       "| open_book_qa   | -       | -       | 95     |\n",
       "| strategy_qa    | 80      | -       | -      |\n",
       "| worldtree      | -       | -       | 99     |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll = coll.filter(lambda x: len(x[\"generated_cot\"])==1)\n",
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll_2 = coll.copy()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge both and filter out where both is present"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = coll_1.merge(coll_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fdd46f26da14d5f9dea275eb1376b75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "244b4ef1c4674865a404d9564b7ec49f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0eb3f76f280240a98ed263466f228637",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2dadd17386a4d028f0b5823e6a2febc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74a0f7262e5d4450956b7b2f4bbf9821",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dd56b44971843c685847104478be765",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 4       | -      |\n",
       "| med_qa         | -       | -       | 9      |\n",
       "| medmc_qa       | -       | 9       | -      |\n",
       "| open_book_qa   | -       | -       | 5      |\n",
       "| strategy_qa    | 12      | -       | -      |\n",
       "| worldtree      | -       | -       | 0      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll = coll.filter(lambda x: len(x[\"generated_cot\"])==2)\n",
    "coll"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "ccbfa654f25866afe66a1c016d0b518a994fbe20a93c2d8b432dbe114020e2d2"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
